package org.fujene.term;

import java.io.CharArrayWriter;
import java.io.IOException;

import org.fujene.term.tool.Escaper;
import org.fujene.term.tool.Optimizer;

public class WordBigramChopper implements Chopper{
	private static final String stopWord = " 　,、.。<＜>＞!！\"”'’(（)）;；:：[「]」{｛}｝-_＿";

	private static final String[] exceptWord = { "although", "also", "am", "an", "and", "are", "as",
			"at", "be", "besides", "by", "can", "else", "for", "in", "into", "is", "may", "moreover", "must",
			"nevertheless", "nor", "of", "on", "onto", "or", "so", "therefore", "this", "that", "the",
			"though", "through", "upon", "with", "without", "yet" };

	private void addStr(TermList strset, String str){
		for(int i = 0; i < exceptWord.length; i++){
			if(str.equals(exceptWord[i])) return;
		}

		strset.add(new Term(str));
	}

	public String standardize(String str){
		// adjust character code to MS932
		str = Escaper.convertSJISToMS932(str);
		// half Kana -> full-size Kana
		str = Optimizer.HanKanaToZen(str);
		// double-size alphabet -> half-size
		str = Optimizer.ZenAlphaToHan(str);
		// unescape
		str = Escaper.unescapeEntityRef(str);
		// half-size symbols -> full-size symbols
		str = Optimizer.HanSymbolToZen(str);
		// capital -> lower
		str = str.toLowerCase();

		return str;
	}

	@Override
	public TermList chop(String content){
		content = standardize(content);

		TermList result = new TermList();

		char[] contchar = content.toCharArray();
		char[] surrogatePair = new char[2];
		char prevKanji = 0x0000;
		CharArrayWriter writer = new CharArrayWriter();
		// Rule: Sequence of numbers and non-CJK words take as one word
		for(int i = 0; i < contchar.length; i++){
			if(stopWord.indexOf(contchar[i]) != -1){
				// stop word
				if(prevKanji != 0x0000){
					addStr(result, Character.toString(prevKanji));
					prevKanji = 0x0000;
				}
				addStr(result, writer.toString());
				writer.reset();
			}else if(contchar[i] <= 0x001F){
				// control characters -> ignore
				if(prevKanji != 0x0000){
					addStr(result, Character.toString(prevKanji));
					prevKanji = 0x0000;
				}
				addStr(result, writer.toString());
				writer.reset();
			}else if(contchar[i] >= 0x0020 && contchar[i] <= 0x007E){
				// ASCII characters
				if(prevKanji != 0x0000){
					addStr(result, writer.toString());
					addStr(result, Character.toString(prevKanji));
					writer.reset();
					prevKanji = 0x0000;
				}
				writer.append(contchar[i]);
			}else if(contchar[i] >= 0x00A1 && contchar[i] <= 0x2E31){
				// other language / symbols
				if(prevKanji != 0x0000){
					addStr(result, writer.toString());
					addStr(result, Character.toString(prevKanji));
					writer.reset();
					prevKanji = 0x0000;
				}
				prevKanji = 0x0000;
				writer.append(contchar[i]);
			}else if(contchar[i] >= 0x3000 && contchar[i] <= 0x3040){
				// other language / symbols
				if(prevKanji != 0x0000){
					addStr(result, writer.toString());
					addStr(result, Character.toString(prevKanji));
					writer.reset();
					prevKanji = 0x0000;
				}
				writer.append(contchar[i]);
			}else if(contchar[i] >= 0x3105 && contchar[i] <= 0x33FF){
				// other language / symbols
				if(prevKanji != 0x0000){
					addStr(result, writer.toString());
					addStr(result, Character.toString(prevKanji));
					writer.reset();
					prevKanji = 0x0000;
				}
				writer.append(contchar[i]);
			}else if(contchar[i] >= 0xA500 && contchar[i] <= 0xABF9){
				// other language / symbols
				if(prevKanji != 0x0000){
					addStr(result, writer.toString());
					addStr(result, Character.toString(prevKanji));
					writer.reset();
					prevKanji = 0x0000;
				}
				writer.append(contchar[i]);
			}else if(contchar[i] >= 0xD800 && contchar[i] <= 0xDBFF){
				// upper surrogate
				if(prevKanji != 0x0000){
					addStr(result, writer.toString());
					addStr(result, Character.toString(prevKanji));
					writer.reset();
					prevKanji = 0x0000;
				}
				surrogatePair[0] = contchar[i];
			}else if(contchar[i] >= 0xDC00 && contchar[i] <= 0xDFFF){
				// lower surrogate
				prevKanji = 0x0000;
				if(surrogatePair[0] >= 0xD800 && surrogatePair[0] <= 0xDBFF){
					surrogatePair[1] = contchar[i];
					try{
						writer.write(surrogatePair);
					}catch(IOException ex){}
				}
			}else if(contchar[i] >= 0xFE00 && contchar[i] <= 0xFE0F){
				// symbols
				if(prevKanji != 0x0000){
					addStr(result, writer.toString());
					addStr(result, Character.toString(prevKanji));
					writer.reset();
					prevKanji = 0x0000;
				}
				writer.append(contchar[i]);
			}else if(contchar[i] >= 0xFFE0 && contchar[i] <= 0xFFFF){
				// symbols
				if(prevKanji != 0x0000){
					addStr(result, writer.toString());
					addStr(result, Character.toString(prevKanji));
					writer.reset();
					prevKanji = 0x0000;
				}
				writer.append(contchar[i]);
			}else if(contchar[i] >= 0xE000 && contchar[i] <= 0xF8FF){
				// private space -> ignore
				if(prevKanji != 0x0000){
					addStr(result, Character.toString(prevKanji));
					prevKanji = 0x0000;
				}
				addStr(result, writer.toString());
				writer.reset();
			}else{
				// bi-gram
				if(writer.size() != 0 && prevKanji == 0x0000){
					// flush buffer
					addStr(result, writer.toString());
					writer.reset();
					// add 1 character
					addStr(result, Character.toString(contchar[i]));
				}else if(prevKanji != 0x0000){
					// add pair
					writer.append(contchar[i]);
					addStr(result, writer.toString());
					writer.reset();
				}else if(writer.size() == 0 && prevKanji == 0x0000){
					// add 1 character
					addStr(result, Character.toString(contchar[i]));
				}
				prevKanji = contchar[i];
				writer.append(contchar[i]);
			}

			if(i == contchar.length - 1){
				// flush buffer
				if(writer.size() != 0) addStr(result, writer.toString());
			}
		}

		return result;
	}
}
